# coding:utf-8

import requests
from bs4 import BeautifulSoup
import random
import urllib


def get_ip_list():
    print("正在获取代理列表...")
    url = 'http://www.xicidaili.com/nn/'
    headers = {
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
        "Accept-Encoding": "gzip, deflate",
        "Accept-Language": "zh-CN,zh;q = 0.9",
        "Cache-Control": "no-cache",
        "Connection": "keep-alive",
        "Cookie": "_free_proxy_session=BAh7B0kiD3Nlc3Npb25faWQGOgZFVEkiJTg0NjlhOWE0MzdmOGU1MzM5ZjMxMzZlYWI5MjQyNDM3BjsAVEkiEF9jc3JmX3Rva2VuBjsARkkiMWhubFRUb0hvU1BCblk4Zk5MbEc2RU9rZmJIUW1ha3hCbmRRUjhlbDhmNnM9BjsARg%3D%3D--9a8103fadf1b3031f75bcad728ec306d5dd45e56; Hm_lvt_0cf76c77469e965d2957f0553e6ecf59=1536646676,1536810485; Hm_lpvt_0cf76c77469e965d2957f0553e6ecf59=1536810985",
        "Host": "www.xicidaili.com",
        "Pragma": "no-cache",
        "Upgrade-Insecure-Requests": "1",
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
    }
    html = requests.get(url=url, headers=headers).text
    soup = BeautifulSoup(html, 'lxml')
    ips = soup.find(id='ip_list').find_all('tr')
    ip_list = []
    for i in range(1, len(ips)):
        ip_info = ips[i]
        tds = ip_info.find_all('td')
        ip_list.append(tds[1].text + ':' + tds[2].text)
    print("代理列表抓取成功.")
    proxy_list = []
    for ip in ip_list:
        proxy_list.append(ip)

    test_url = "http://ip.chinaz.com/getip.aspx"
    for proxyIp in proxy_list[0:100]:
        try:
            proxy = urllib.request.ProxyHandler({'http': proxyIp})  # 设置proxy
            opener = urllib.request.build_opener(proxy)  # 挂载opener
            urllib.request.install_opener(opener)  # 安装opener
            page = opener.open(test_url,timeout=3).read()
            ip_list.append(proxyIp)
        except Exception as e:
            print(proxyIp)
            print(e)
            continue
    return ip_list


def get_random_ip(ip_list):
    print("正在设置随机代理...")
    proxy_list = []
    for ip in ip_list:
        proxy_list.append('http://' + ip)
    proxy_ip = random.choice(proxy_list)
    proxies = {'http': proxy_ip}
    print("代理设置成功.")
    return proxies


ip_list = get_ip_list()
ips = get_random_ip(ip_list)
print(ips)
